#!/usr/bin/env python
# -*- coding:utf-8 -*-
import subprocess
import json
import sys
import re
import os
import psutil
import time
import requests
from copy import deepcopy
from datetime import datetime, timedelta
from pybloom_live import ScalableBloomFilter
from secret import batch_encrypt_decrypt_dict
from aes import contact_aes, name_aes
from apscheduler.schedulers.blocking import BlockingScheduler
from datastore.mongodb import MongoDBClientWrapper

reload(sys)
sys.setdefaultencoding('utf-8')


class ExportAndTj(object):
    def __init__(self, file_name):
        self.db = MongoDBClientWrapper()
        # 脚本开始时间
        self.start_time = datetime.now().strftime('%Y-%m-%d %H:%M')
        self.file_name = file_name
        self.init_file(file_name)
        self.write_log_f.write('***  the start {} memory_percent：{}***\n'.format(datetime.now(), self.memory_check()))
        # 让开发者立即看到脚本效果
        self.write_log_f.flush()
        self.init_zy_bloom()

    def init_zy_bloom(self):
        file_name_ls = []
        for file_name in os.listdir('/mnt5/dsf'):
            if file_name.count('dsf_zy_qy_nameId_get_new') and file_name.count('dc'):
                file_name_ls.append(file_name)
        zy_file_name = '/mnt5/dsf/{}'.format(max(file_name_ls))

        self.zy_nameId_bloom = self.bloom_filter_fac(mode=4)
        index = 0
        with open(zy_file_name, 'r') as f:
            self.write_log_f.write('正在读取{}  {}\n'.format(zy_file_name, index))
            self.write_log_f.flush()
            for line in f:
                line = line.strip()
                self.zy_nameId_bloom.add(line)
                index += 1
                if index % 10000 == 0:
                    self.write_log_f.write('正在读取{}  {}\n'.format(zy_file_name,index))
                    self.write_log_f.flush()
                # if index > 10000:
                #     break
        self.write_log_f.write('在营企业共有{}\n'.format(index))
        self.write_log_f.flush()

    def bloom_filter_fac(self,initial_capacity=1000000, error_rate=0.001, mode=2):
        '''统一控制bloom配置'''
        return ScalableBloomFilter(initial_capacity=initial_capacity, error_rate=error_rate, mode=mode)  # 慢点但是省内存

    def init_file(self, file_name, mod='w'):
        '''初始化需要写入的文件'''
        dir_path = file_name.rsplit('/', 1)[0]
        if not os.path.exists(dir_path):
            os.makedirs(dir_path)
        self.file_dir_path = dir_path
        self.write_log_f = open('{}_log'.format(file_name), mod)
        self.write_dc_f = open('{}_dc'.format(file_name), mod)
        self.write_tj_f = open('{}_tj'.format(file_name), mod)
        self.write_error_f = open('{}_error'.format(file_name), mod)

    @staticmethod
    def get_date_range(days=7):
        '''取一个时间范围'''
        today = datetime.now()
        one_day = timedelta(days=days)
        yesterday = today - one_day
        _min = datetime(year=yesterday.year, month=yesterday.month, day=yesterday.day)
        _max = datetime(year=today.year, month=today.month, day=today.day)
        return _min, _max

    def get_records_crusor(self):
        '''获得查询结果cursor对象'''
        col = self.db.p_col('contact')
        filter_ = {
            'contactType': {"$in": [0, 1, 2]},
        }

        projection = ["nameId", "isBaiduContact", "contact", "contactType", "contactSource", "import_update_time",
                      "create_time", "contactChannelList"]
        # 0 mobile 手机 ； 1 phone 固话 2:邮箱
        records_crusor = col.find(filter=filter_, projection=projection)#.sort([("$natural", -1)])
        # todo
        # records_crusor = col.find(filter=filter_, projection=projection).limit(500000).sort([("$natural", -1)])
        return records_crusor

    def get_data_from_mongo(self):
        '''从mongo中查出数据'''
        records_crusor = self.get_records_crusor()
        source_dic = {
            'qcc': ['qcc'],
            'b2b': ['jc001','027zpw','cnlist','hqew','byxx', 'N127', 'mmfj', 'tonbao', '21food', 'jx188', 'tpy888', 'qiugouxinxi', 'guidechem',
                    'made-in-china', 'tfsb', 'zhaozhaoqu', 'czvv', '52bjw', 'huohuli', '3g.ganji', 'booksir', '78gk',
                    'chadedao', 'baixing', '11467', 'sonhoo', 'd17', 'alibaba', '51sole', 'qydsj', 'cn.made-in-china',
                    'm.cpooo', 'qymgc', 'yellowurl', 'makepolo', 'cnlinfo', 'bestb2b', '51g3', 'jiqimao', 'qiyetong',
                    'cn5135', 'xianjichina', 'company.biz72', 'chem17', 'bzjw', 'qy6', 'globalimporter', 'c-c', 'abiz',
                    'sooshong', 'jqw', 'wmb2b', 'eb80', '1024sj', 'yiwugo', 'bizrobot', 'qichamao', 'herostart',
                    'jiancai365', 'baidu_alibaba', 'chinafastenerinfo', 'b2b168', 'globrand', '71ab', 'vooec',
                    'tecenet', '912688', 'ic98', 'huangye88', '258', 'huijucn', 'qqhyw', 'youboy', 'texindex', 'qiyeku',
                    'hunt007', 'gtobal', 'jinanfa', 'b2b.baidu', 'hc360', 'qth58', '007swz', 'ebdoor', 'taojindi',
                    'biz72', 'mobile.gongchang', 'bmlink', '7wsh', '99114', '58', 'company.dzsc', 'ilsnc', u'atobo',
                    '1688', 'ccn', 'trustexporter', 'sg560', 'guihenongye', 'coovee', 'zk71', 'gbs', 'zj123'],
            'bidding': ['ccgp', 'ggzy', 'zhaotx', 'ggzyfw.beijing', 'ggzy.gz', 'ggzy.sz', 'qudaobao', 'zfcg.sh',
                        'gdgpo', 'zfcg.czt.zj', 'ccgp-sichuan.gov', 'jxsggzy', 'ccgp-shandong', 'ccgp-hainan',
                        'sxzwfw.gov', 'ccgp-chongqing', 'ccgp-tianjin', 'ccgp-shaanxi', 'ccgp-jiangsu', 'ccgp-guangxi',
                        'hngp', 'ccgp-hunan', 'ccgp-liaoning', 'qhggzyjy', 'nxggzyjy', 'hbggzyfwpt', 'ccgp-yunnan',
                        'nmgp', 'xzzbtb', 'ggzyfw.fujian', 'ccgp-anhui', 'ccgp-hebei', 'ggzy.guizhou', 'zwfw.xinjiang',
                        'ggzyjy.gansu', 'jl.gov', 'hljcg', 'gdpepep', 'ebnew', 'vip', 'hunan.zcygov.cn', 'gdgpo.czt.gd',
                        'gd-prod.oss-cn-beijing.aliyuncs.com', 'zjcs.gdggzy.org.cn', 'ec.chng.com.cn', 'obei.com.cn',
                        'ccgp-xinjiang.gov.cn', 'ec.mcc.com.cn', 'jswater.jiangsu.gov.cn'],
            'annual_report': 'annual_report',
            # ['annual_report_1997', 'annual_report_1998', 'annual_report_1999', 'annual_report_2000', 'annual_report_2001', 'annual_report_2002', 'annual_report_2003', 'annual_report_2004', 'annual_report_2005', 'annual_report_2006', 'annual_report_2007', 'annual_report_2008', 'annual_report_2009', 'annual_report_2010', 'annual_report_2011', 'annual_report_2012', 'annual_report_2013', 'annual_report_2014', 'annual_report_2015', 'annual_report_2016', 'annual_report_2017', 'annual_report_2018', 'annual_report_2019', 'annual_report_2020', 'annual_report_2021'],
            'gw': 'tungee_homepage',
            'zp': ["58guakao", "baiduzhaopin", "baidajob", "51job", "goodjob", "lagou", "neitui", "jobcn", "360guakao",
                   "chinahr", "hunt007", "liepin", "job36", "zhaopin", "ganji", "528", "dajie", "jobui", "bole", "51rc",
                   "zhilian", "zhipin", "58", "bendizhaopin", "98pz", "fengpintech", "jobeast", "yilan", "nowcoder",
                   "78gk","job1001"]

        }

        result_dic = {
            'all_table_count': 0,  # 库内手机总量(未去重)
            'all_update': 0,  # 周更新量
            'all_add': 0,  # 周新增量 (未去重)
            'all_abs_add': 0,  # 绝对新增(contactChannelList为1的)

            'mobile_update': 0,  # 手机更新数量
            'mobile_update_dup': self.bloom_filter_fac(),  # 去重后手机更新数量的企业数量
            'mobile_add': 0, # (未去重)
            'mobile_add_dup': self.bloom_filter_fac(),
            'mobile_abs_add': 0,

            'phone_update': 0,
            'phone_update_dup': self.bloom_filter_fac(),
            'phone_add': 0,
            'phone_add_dup': self.bloom_filter_fac(),
            'phone_abs_add': 0,

            'email_update': 0,
            'email_update_dup': self.bloom_filter_fac(),
            'email_add': 0,
            'email_add_dup': self.bloom_filter_fac(),
            'email_abs_add': 0,
            # -------------新增--------
            # 可能存在手机号重复的情况，即多家企业拥有同一个手机号，以后再对手机号进行去重
            "one_month_mobile_update": 0,  # 近一个月手机更新数量
            "one_month_mobile_update_dup": self.bloom_filter_fac(),  # 近一个月手机更新数量映射企业去重数
            "one_month_phone_update": 0,
            "one_month_phone_update_dup": self.bloom_filter_fac(),
            "one_month_email_update": 0,
            "one_month_email_update_dup": self.bloom_filter_fac(),
            "two_month_mobile_update": 0,  # 近2个月手机更新数量
            "two_month_mobile_update_dup": self.bloom_filter_fac(),  # 近2个月手机更新数量映射企业去重数
            "two_month_phone_update": 0,
            "two_month_phone_update_dup": self.bloom_filter_fac(),
            "two_month_email_update": 0,
            "two_month_email_update_dup": self.bloom_filter_fac(),
            "three_month_mobile_update": 0,  # 近3个月手机更新数量 TODO
            "three_month_mobile_update_dup": self.bloom_filter_fac(),  # 近3个月手机更新数量映射企业去重数TODO
            "three_month_phone_update": 0,  # TODO
            "three_month_phone_update_dup": self.bloom_filter_fac(),  # TODO
            "three_month_email_update": 0,  # TODO
            "three_month_email_update_dup": self.bloom_filter_fac(),  # TODO
            # 在营总量统计
            "zy_enterprise_count": self.zy_nameId_bloom.count,  # 在营企业的数量
            "zy_enterprise_mobile_count": 0,  # 在营企业的手机数量（未去重）  TODO
            "week_zy_enterprise_mobile_update": 0,  # 在营企业的手机数量周更新
            "week_zy_enterprise_mobile_add": 0,  # 在营企业的手机数量周新增
            "week_zy_enterprise_mobile_abs_add": 0,  # 在营企业的手机数量周绝对新增
            "one_month_zy_enterprise_mobile_update": 0,  # 在营企业的手机数量近一个月更新
            "two_month_zy_enterprise_mobile_update": 0,
            "three_month_zy_enterprise_mobile_update": 0,
            "zy_enterprise_phone_count": 0,  # TODO
            "week_zy_enterprise_phone_update": 0,
            "week_zy_enterprise_phone_add": 0,
            "week_zy_enterprise_phone_abs_add": 0,
            "one_month_zy_enterprise_phone_update": 0,  # 在营企业的手机数量近一个月更新
            "two_month_zy_enterprise_phone_update": 0,
            "three_month_zy_enterprise_phone_update": 0,
            "zy_enterprise_email_count": 0,  # TODO
            "week_zy_enterprise_email_update": 0,
            "week_zy_enterprise_email_add": 0,
            "week_zy_enterprise_email_abs_add": 0,
            "one_month_zy_enterprise_email_update": 0,  # 在营企业的手机数量近一个月更新
            "two_month_zy_enterprise_email_update": 0,
            "three_month_zy_enterprise_email_update": 0,
            # 全表数量统计
            "table_mobile_count": self.bloom_filter_fac(),  # 库内手机总量去重 TODO
            'table_mobile_count_dup': self.bloom_filter_fac(),  # 库内手机总量 对应的 去重企业数量 TODO
            "table_phone_count": self.bloom_filter_fac(),  # 库内固话总量  TODO
            'table_phone_count_dup': self.bloom_filter_fac(),  # 库内固话总量的去重企业数量  TODO
            "table_email_count": self.bloom_filter_fac(),  # 库内邮箱总量 # TODO
            'table_email_count_dup': self.bloom_filter_fac(),  # 库内邮箱总量的去重企业数量 # TODO

            'source_result_dic': {
                'qcc': {
                    'mobile_add': 0,  # 企查查周新增手机
                    'mobile_abs_add': 0,  # 企查查周绝对新增手机
                    'zy_mobile_add': 0,  # 在营企业新增手机
                    'zy_mobile_abs_add': 0,  # 在营企业周绝对新增手机
                },
                'annual_report': {
                    'mobile_add': 0,
                    'mobile_abs_add': 0,
                    'zy_mobile_add': 0,
                    'zy_mobile_abs_add': 0,
                },
                'baidu': {
                    'mobile_add': 0,
                    'mobile_abs_add': 0,
                    'zy_mobile_add': 0,
                    'zy_mobile_abs_add': 0,
                },
                'b2b': {
                    'mobile_add': 0,
                    'mobile_abs_add': 0,
                    'zy_mobile_add': 0,
                    'zy_mobile_abs_add': 0,
                },
                'bidding': {
                    'mobile_add': 0,
                    'mobile_abs_add': 0,
                    'zy_mobile_add': 0,
                    'zy_mobile_abs_add': 0,
                },
                'gw': {
                    'mobile_add': 0,
                    'mobile_abs_add': 0,
                    'zy_mobile_add': 0,
                    'zy_mobile_abs_add': 0,
                },
                'zp': {
                    'mobile_add': 0,
                    'mobile_abs_add': 0,
                    'zy_mobile_add': 0,
                    'zy_mobile_abs_add': 0,
                },
                # 'other_source_set': set()
            },

        }
        week_min, week_max = self.get_date_range(days=7)
        one_month_min, one_month_max = self.get_date_range(days=30)
        two_month_min, two_month_max = self.get_date_range(days=60)
        three_month_min, three_month_max = self.get_date_range(days=90)
        self.log('search mongo end ...\n')
        for index, record_dic in enumerate(records_crusor):
            memory_percent = self.memory_check()
            if index == 0:
                log_info = 'find is ok! memory_percent：{}'.format(memory_percent)
                self.log(log_info)
            contact = record_dic.get('contact')
            contactType = record_dic.get('contactType')
            isBaiduContact = record_dic.get('isBaiduContact')
            nameId = record_dic.get('nameId')
            update_time = record_dic.get('import_update_time', '')
            create_time = record_dic.get('create_time', '')
            contactChannelList = record_dic.get('contactChannelList', [])
            contactSource = record_dic.get('contactSource', '').strip() or contactChannelList and contactChannelList[-1]['contactSource']
            result_dic['all_table_count'] += 1
            if not all([update_time, create_time,contactChannelList]):
                self.write_error_f.write('{}\t{} jump\n'.format(nameId,contact))
                continue
            # 是否是绝对 TODO 判断逻辑不严谨，后期改成先遍历以前的联系方式加入bloom
            abs_flag = len(contactChannelList) == 1
            if abs_flag:
                result_dic['all_abs_add'] += 1
            zy_flag = nameId in self.zy_nameId_bloom
            # 库内联系方式与映射企业数去重
            if contactType == 0:
                result_dic['table_mobile_count'].add(contact)
                # 以前不存在返回True，否则False
                is_mobile_dup = result_dic['table_mobile_count_dup'].add(nameId)
                if zy_flag:
                    result_dic['zy_enterprise_mobile_count'] += 1
                if one_month_min <= update_time <= one_month_max:
                    result_dic['one_month_mobile_update'] += 1
                    result_dic['one_month_mobile_update_dup'].add(nameId)
                    result_dic['two_month_mobile_update'] += 1
                    result_dic['two_month_mobile_update_dup'].add(nameId)
                    result_dic['three_month_mobile_update'] += 1
                    result_dic['three_month_mobile_update_dup'].add(nameId)
                    if zy_flag:
                        result_dic['one_month_zy_enterprise_mobile_update'] += 1
                        result_dic['two_month_zy_enterprise_mobile_update'] += 1
                        result_dic['three_month_zy_enterprise_mobile_update'] += 1
                elif two_month_min <= update_time <= two_month_max:
                    result_dic['two_month_mobile_update'] += 1
                    result_dic['two_month_mobile_update_dup'].add(nameId)
                    result_dic['three_month_mobile_update'] += 1
                    result_dic['three_month_mobile_update_dup'].add(nameId)
                    if zy_flag:
                        result_dic['two_month_zy_enterprise_mobile_update'] += 1
                        result_dic['three_month_zy_enterprise_mobile_update'] += 1
                elif three_month_min <= update_time <= three_month_max:
                    result_dic['three_month_mobile_update'] += 1
                    result_dic['three_month_mobile_update_dup'].add(nameId)
                    if zy_flag:
                        result_dic['three_month_zy_enterprise_mobile_update'] += 1
                if week_min <= create_time < week_max:
                    result_dic['all_add'] += 1
                    result_dic['mobile_add'] += 1
                    result_dic['mobile_add_dup'].add(nameId)
                    if zy_flag:
                        result_dic['week_zy_enterprise_mobile_add'] += 1
                    if isBaiduContact:
                        source_key = 'baidu'
                    elif contactSource in source_dic['qcc']:
                        source_key = 'qcc'
                    elif contactSource.startswith(source_dic['annual_report']):
                        source_key = 'annual_report'
                    elif contactSource.startswith(source_dic['gw']):
                        source_key = 'gw'
                    elif contactSource in source_dic['b2b']:
                        source_key = 'b2b'
                    elif contactSource in source_dic['bidding']:
                        source_key = 'bidding'
                    elif contactSource in source_dic['zp']:
                        source_key = 'zp'
                    else:
                        source_key = 'other'
                        # if contactSource:
                        #     source_key = contactSource
                        # else:
                        #     source_key = contactChannelList[-1]['contactSource']
                    if source_key not in result_dic['source_result_dic']:
                        result_dic['source_result_dic'][source_key] = {'mobile_add': 0, 'mobile_abs_add': 0,
                                                                       'zy_mobile_add': 0, 'zy_mobile_abs_add': 0}
                    result_dic['source_result_dic'][source_key]['mobile_add'] += 1
                    if zy_flag:
                        result_dic['source_result_dic'][source_key]['zy_mobile_add'] += 1
                    if abs_flag:
                        result_dic['mobile_abs_add'] += 1
                        result_dic['source_result_dic'][source_key]['mobile_abs_add'] += 1
                        if zy_flag:
                            result_dic['week_zy_enterprise_mobile_abs_add'] += 1
                            result_dic['source_result_dic'][source_key]['zy_mobile_abs_add'] += 1
                # 不是昨日更新 而是昨日更新数
                elif week_min <= update_time < week_max:
                    result_dic['all_update'] += 1
                    result_dic['mobile_update'] += 1
                    result_dic['mobile_update_dup'].add(nameId)
                    if zy_flag:
                        result_dic['week_zy_enterprise_mobile_update'] += 1
            elif contactType == 1:
                result_dic['table_phone_count'].add(contact)
                is_phone_dup = result_dic['table_phone_count_dup'].add(nameId)
                if zy_flag:
                    result_dic['zy_enterprise_phone_count'] += 1
                if one_month_min <= update_time <= one_month_max:
                    result_dic['one_month_phone_update'] += 1
                    result_dic['one_month_phone_update_dup'].add(nameId)
                    result_dic['two_month_phone_update'] += 1
                    result_dic['two_month_phone_update_dup'].add(nameId)
                    result_dic['three_month_phone_update'] += 1
                    result_dic['three_month_phone_update_dup'].add(nameId)
                    if zy_flag:
                        result_dic['one_month_zy_enterprise_phone_update'] += 1
                        result_dic['two_month_zy_enterprise_phone_update'] += 1
                        result_dic['three_month_zy_enterprise_phone_update'] += 1
                elif two_month_min <= update_time <= two_month_max:
                    result_dic['two_month_phone_update'] += 1
                    result_dic['two_month_phone_update_dup'].add(nameId)
                    result_dic['three_month_phone_update'] += 1
                    result_dic['three_month_phone_update_dup'].add(nameId)
                    if zy_flag:
                        result_dic['two_month_zy_enterprise_phone_update'] += 1
                        result_dic['three_month_zy_enterprise_phone_update'] += 1
                elif three_month_min <= update_time <= three_month_max:
                    result_dic['three_month_phone_update'] += 1
                    result_dic['three_month_phone_update_dup'].add(nameId)
                    if zy_flag:
                        result_dic['three_month_zy_enterprise_phone_update'] += 1
                if week_min <= create_time < week_max:
                    result_dic['all_add'] += 1
                    result_dic['phone_add'] += 1
                    result_dic['phone_add_dup'].add(nameId)
                    if zy_flag:
                        result_dic['week_zy_enterprise_phone_add'] += 1
                    if abs_flag:
                        result_dic['phone_abs_add'] += 1
                        if zy_flag:
                            result_dic['week_zy_enterprise_phone_abs_add'] += 1
                # 不是昨日更新 而是昨日更新数
                elif week_min <= update_time < week_max:
                    result_dic['all_update'] += 1
                    result_dic['phone_update'] += 1
                    result_dic['phone_update_dup'].add(nameId)
                    if zy_flag:
                        result_dic['week_zy_enterprise_phone_update'] += 1
            elif contactType == 2:
                result_dic['table_email_count'].add(contact)
                is_email_dup = result_dic['table_email_count_dup'].add(nameId)
                if zy_flag:
                    result_dic['zy_enterprise_email_count'] += 1
                if one_month_min <= update_time <= one_month_max:
                    result_dic['one_month_email_update'] += 1
                    result_dic['one_month_email_update_dup'].add(nameId)
                    result_dic['two_month_email_update'] += 1
                    result_dic['two_month_email_update_dup'].add(nameId)
                    result_dic['three_month_email_update'] += 1
                    result_dic['three_month_email_update_dup'].add(nameId)
                    if zy_flag:
                        result_dic['one_month_zy_enterprise_email_update'] += 1
                        result_dic['two_month_zy_enterprise_email_update'] += 1
                        result_dic['three_month_zy_enterprise_email_update'] += 1
                elif two_month_min <= update_time <= two_month_max:
                    result_dic['two_month_email_update'] += 1
                    result_dic['two_month_email_update_dup'].add(nameId)
                    result_dic['three_month_email_update'] += 1
                    result_dic['three_month_email_update_dup'].add(nameId)
                    if zy_flag:
                        result_dic['two_month_zy_enterprise_email_update'] += 1
                        result_dic['three_month_zy_enterprise_email_update'] += 1
                elif three_month_min <= update_time <= three_month_max:
                    result_dic['three_month_email_update'] += 1
                    result_dic['three_month_email_update_dup'].add(nameId)
                    if zy_flag:
                        result_dic['three_month_zy_enterprise_email_update'] += 1
                if week_min <= create_time < week_max:
                    result_dic['all_add'] += 1
                    result_dic['email_add'] += 1
                    result_dic['email_add_dup'].add(nameId)
                    if zy_flag:
                        result_dic['week_zy_enterprise_email_add'] += 1
                    if abs_flag:
                        result_dic['email_abs_add'] += 1
                        if zy_flag:
                            result_dic['week_zy_enterprise_email_abs_add'] += 1
                # 不是昨日更新 而是昨日更新数
                elif week_min <= update_time < week_max:
                    result_dic['all_update'] += 1
                    result_dic['email_update'] += 1
                    result_dic['email_update_dup'].add(nameId)
                    if zy_flag:
                        result_dic['week_zy_enterprise_email_update'] += 1
            if index % 10000 == 0:
                if index % 1000000 == 0:
                    self.save_tj_info(deepcopy(result_dic))
                log_info = '{} — index：{}\tmemory_percent:{}'.format(self.file_name, index, memory_percent)
                self.log(log_info)
        else:
            self.save_tj_info(deepcopy(result_dic))
        return result_dic

    def save_tj_info(self, result_dic):
        for k, v in result_dic.items():
            if isinstance(v, ScalableBloomFilter):
                result_dic[k] = v.count
        self.write_tj_f.write('{}\n\n\n\n'.format(json.dumps(result_dic)))
        self.write_tj_f.flush()
        self.write_log_f.write('写入一次tj文件 {}\n'.format(json.dumps(result_dic)))

    def run_dc(self):
        '''运行非去重计数统计'''
        result_dic = self.get_data_from_mongo()
        mk_text = self.make_mk_text(result_dic)
        self.send_msg_to_dd(mk_text)
        self.close_file()

    def log(self, log_info):
        ''''''
        current_time = datetime.now().strftime('%Y-%m-%d %H:%M')
        log_info += '\tnow：{}\n'.format(current_time)
        self.write_log_f.write(log_info)
        self.write_log_f.flush()

    def make_mk_text(self, result_dic):
        '''通过数据 拼接出mk_text'''
        # mk_text = ''
        # all_update = v.get('all_update', 0)
        # all_add = v.get('all_add', 0)
        # all_abs_add = v.get('all_abs_add', 0)
        #
        # mobile_update = v.get('mobile_update', 0)
        # mobile_add = v.get('mobile_add', 0)
        # mobile_update_dup = len(v.get('mobile_update_dup', ''))
        # mobile_add_dup = len(v.get('mobile_add_dup', ''))
        # mobile_abs_add = v.get('mobile_abs_add', 0)
        #
        # phone_update = v.get('phone_update', 0)
        # phone_add = v.get('phone_add', 0)
        # phone_update_dup = len(v.get('phone_update_dup', ''))
        # phone_add_dup = len(v.get('phone_add_dup', ''))
        # phone_abs_add = v.get('phone_abs_add', 0)
        #
        # email_update = v.get('email_update', 0)
        # email_add = v.get('email_add', 0)
        # email_update_dup = len(v.get('email_update_dup', ''))
        # email_add_dup = len(v.get('email_add_dup', ''))
        # email_abs_add = v.get('email_abs_add', 0)

        # _min, _max = self.get_date_range()
        # _min_date = _min.strftime('%Y-%m-%d')
        # _max_date = _max.strftime('%Y-%m-%d')
        mk_text = ''' 
                **库内手机总量：{}**  
                **映射企业数量（去重）：{}**  
                **近一个月更新：{}**  
                **去重企业：{}**  
                **近两个月更新：{}**  
                **去重企业：{}**  
                **近三个月更新：{}**  
                **去重企业：{}**  
                **周更新：{}**  
                **去重企业：{}**  
                **周新增：{}**  
                **去重企业：{}**  
                **周绝对新增：{}**  
                **==========**  
                **库内固话总量：{}**  
                **映射企业数量（去重）：{}**  
                **近一个月更新：{}**  
                **去重企业：{}**  
                **近两个月更新：{}**  
                **去重企业：{}**  
                **近三个月更新：{}**  
                **去重企业：{}**  
                **周更新：{}**  
                **去重企业：{}**  
                **周新增：{}**  
                **去重企业：{}**  
                **周绝对新增：{}**  
                **==========**  
                **库内邮箱总量：{}**  
                **映射企业数量（去重）：{}**  
                **近一个月更新：{}**  
                **去重企业：{}**  
                **近两个月更新：{}**  
                **去重企业：{}**  
                **近三个月更新：{}**  
                **去重企业：{}**  
                **周更新：{}**  
                **去重企业：{}**  
                **周新增：{}**  
                **去重企业：{}**  
                **周绝对新增：{}**  
                **==========**  
                **在营企业数量：{}**  
                **在营企业手机数量：{}**  
                **近一个月更新：{}**  
                **近两个月更新：{}**  
                **近三个月更新：{}**  
                **周更新：{}**  
                **周新增：{}**  
                **周绝对新增：{}**  
                **==========**  
                **在营企业固话数量：{}**  
                **近一个月更新：{}**  
                **近两个月更新：{}**  
                **近三个月更新：{}**  
                **周更新：{}**  
                **周新增：{}**  
                **周绝对新增：{}**  
                **==========**  
                **在营企业邮箱数量：{}**  
                **近一个月更新：{}**  
                **近两个月更新：{}**  
                **近三个月更新：{}**  
                **周更新：{}**  
                **周新增：{}**  
                **周绝对新增：{}**  
                **==========**  
                '''.format(result_dic['table_mobile_count'].count, result_dic['table_mobile_count_dup'].count,
                           result_dic['one_month_mobile_update'], result_dic['one_month_mobile_update_dup'].count,
                           result_dic['two_month_mobile_update'], result_dic['two_month_mobile_update_dup'].count,
                           result_dic['three_month_mobile_update'], result_dic['three_month_mobile_update_dup'].count,
                           result_dic['mobile_update'], result_dic['mobile_update_dup'].count,
                           result_dic['mobile_add'], result_dic['mobile_add_dup'].count, result_dic['mobile_abs_add'],

                           result_dic['table_phone_count'].count, result_dic['table_phone_count_dup'].count,
                           result_dic['one_month_phone_update'], result_dic['one_month_phone_update_dup'].count,
                           result_dic['two_month_phone_update'], result_dic['two_month_phone_update_dup'].count,
                           result_dic['three_month_phone_update'], result_dic['three_month_phone_update_dup'].count,
                           result_dic['phone_update'], result_dic['phone_update_dup'].count,
                           result_dic['phone_add'], result_dic['phone_add_dup'].count, result_dic['phone_abs_add'],

                           result_dic['table_email_count'].count, result_dic['table_email_count_dup'].count,
                           result_dic['one_month_email_update'], result_dic['one_month_email_update_dup'].count,
                           result_dic['two_month_email_update'], result_dic['two_month_email_update_dup'].count,
                           result_dic['three_month_email_update'], result_dic['three_month_email_update_dup'].count,
                           result_dic['email_update'], result_dic['email_update_dup'].count,
                           result_dic['email_add'], result_dic['email_add_dup'].count, result_dic['email_abs_add'],

                           result_dic['zy_enterprise_count'],
                           result_dic['zy_enterprise_mobile_count'],
                           result_dic['one_month_zy_enterprise_mobile_update'],
                           result_dic['two_month_zy_enterprise_mobile_update'],
                           result_dic['three_month_zy_enterprise_mobile_update'],
                           result_dic['week_zy_enterprise_mobile_update'],
                           result_dic['week_zy_enterprise_mobile_add'],
                           result_dic['week_zy_enterprise_mobile_abs_add'],

                           result_dic['zy_enterprise_phone_count'],
                           result_dic['one_month_zy_enterprise_phone_update'],
                           result_dic['two_month_zy_enterprise_phone_update'],
                           result_dic['three_month_zy_enterprise_phone_update'],
                           result_dic['week_zy_enterprise_phone_update'], result_dic['week_zy_enterprise_phone_add'],
                           result_dic['week_zy_enterprise_phone_abs_add'],

                           result_dic['zy_enterprise_email_count'],
                           result_dic['one_month_zy_enterprise_email_update'],
                           result_dic['two_month_zy_enterprise_email_update'],
                           result_dic['three_month_zy_enterprise_email_update'],
                           result_dic['week_zy_enterprise_email_update'], result_dic['week_zy_enterprise_email_add'],
                           result_dic['week_zy_enterprise_email_abs_add'],
                           )
        for key,v in result_dic['source_result_dic'].items():
            mk_text += '''  
                **==========**  
                **来源：{}**  
                **周绝对新增(手机)：{}**  
                **周新增(手机)：{}**  
                **在营企业手机周绝对新增：{}**  
                **在营企业手机周新增：{}**  
                **==========**  
                '''.format(key,v['mobile_abs_add'],v['mobile_add'],
                           v['zy_mobile_abs_add'], v['zy_mobile_add'])
        with open('周报统计','w') as f:
            f.write(mk_text)
        # mk_text = '''**{}至{}**
        # **总更新量：{}**
        # **手机：{}**
        # **去重企业：{}**
        # **固话：{}**
        # **去重企业：{}**
        # **邮箱：{}**
        # **去重企业：{}**
        # **总新增量：{}**
        # **手机：{}**
        # **去重企业：{}**
        # **固话：{}**
        # **去重企业：{}**
        # **邮箱：{}**
        # **去重企业：{}**
        # **绝对新增量：{}**
        # **手机：{}**
        # **固话：{}**
        # **邮箱：{}**
        # '''.format(_min_date, _max_date,
        #            all_update,
        #            mobile_update, mobile_update_dup,
        #            phone_update, phone_update_dup,
        #            email_update, email_update_dup,
        #            all_add,
        #            mobile_add, mobile_add_dup,
        #            phone_add, phone_add_dup,
        #            email_add, email_add_dup,
        #            all_abs_add, mobile_abs_add, phone_abs_add, email_abs_add
        #            ) + mk_text
        return mk_text

    def send_msg_to_dd(self, mk_text, title='联系方式周报数据监控'):
        '''发送mk_text到钉钉'''
        # 联系方式测试机器人
        # web_hook_url = 'https://oapi.dingtalk.com/robot/send?access_token=1c23bf41ce0e5a7b453129f719c710e395eeb50445a190212b2eb04c109d5490'
        # 联系方式群正式机器人
        # web_hook_url = 'https://oapi.dingtalk.com/robot/send?access_token=6a732fad118e7d46b1662b2c9a623591e1441e637de8ae9fb528d30ce449f2ba'
        # 个人测试群
        # web_hook_url = 'https://oapi.dingtalk.com/robot/send?access_token=d7e420a71081174b3eac4df759d6cc78f7ead8557d3ba3af20883613167597d0'
        # 联系方式群
        web_hook_url = 'https://oapi.dingtalk.com/robot/send?access_token=4775d7395e0932c4938ae80416137c5313ac6f994112c7e1519035cbfdea6679'
        msg_body = {
            "msgtype": "markdown",
            "markdown": {
                "title": title,
                "text": mk_text
            },
        }
        headers = {
            "Content-Type": "application/json;charset=utf-8"
        }
        retry_count = 0
        while retry_count < 10:
            try:
                response = requests.post(url=web_hook_url, data=json.dumps(msg_body), headers=headers)
                if response.status_code == 200:
                    self.write_log_f.write("{} 发送钉钉信息成功：{}".format(datetime.now(), response.status_code))

                    break
                else:
                    self.write_log_f.write("{}  返回状态码不正常：{}".format(datetime.now(), response.status_code))
            except Exception as e:
                self.write_log_f.write("{}  请求异常：{}".format(datetime.now(), e))
                retry_count += 1
                time.sleep(2)

    def close_file(self):
        '''关闭文件流'''
        # 关闭所有文件
        self.write_log_f.write('\n***  the end {} memory_percent：{}***\n'.format(datetime.now(), self.memory_check()))
        self.write_log_f.close()
        self.write_dc_f.close()
        self.write_tj_f.close()
        self.write_error_f.close()
        # if '/mnt5/log' in self.file_dir_path:
        #     shell_cmd = "rm -r {}/*dup".format(self.file_dir_path)
        #     p = subprocess.Popen(shell_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        #     result_str = p.stdout.read().decode('utf-8')

    def memory_check(self, max_percent=97):
        '''内存检测'''
        memory_info = psutil.virtual_memory()
        memory_percent = memory_info.percent
        while memory_percent > max_percent:  # 内存占比
            self.write_log_f.write('内存情况:{}\tnow:{}\n'.format(memory_percent, datetime.now()))
            time.sleep(300)
            memory_info = psutil.virtual_memory()
            memory_percent = memory_info.percent
        return memory_percent


def run():
    today_str = str(datetime.now().date())  # '2021-05-28'
    file_name = "/mnt5/dsf/周报_" + today_str + __file__.rsplit('.', 1)[0]
    export_and_tj = ExportAndTj(file_name)
    export_and_tj.run_dc()

def scheduler_run():
    scheduler = BlockingScheduler()
    scheduler.add_job(run, 'cron', day_of_week="sun", hour='1', id='dd_run')
    # scheduler.add_job(run, 'interval',seconds=10)

    scheduler.start()

if __name__ == '__main__':
    run()
    # scheduler_run()

